import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

#读取数据
message=pd.read_csv('data/sms_spam.csv',names=['label','message'])
# print(message.head())
message=message[1:]
# print(message)
print(message.groupby('label').describe())
print(len(message))
#加入文本长度属性
message['length']=message['message'].apply(len)
# print(message.head())
#以长度为变量通过直方图统计频率
# message['length'].plot(bins=100,kind='hist')
# plt.show()
#以label为区别对比文本长度与垃圾文件之间的关系
# message.hist(column='length',by='lable',bins=50)
# plt.show()

import string
from nltk.corpus import stopwords
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report
#对样本经行分割
msg_train, msg_test, label_train, label_test = \
    train_test_split(message['message'], message['label'], test_size=0.2)
#过滤单词的方法
def text_process(text):
    nopunc=[char for char in text if char not in string.punctuation]#去除掉句中的标点符号
    nopunc = ''.join(nopunc)#重新组合
    nopunc= [word for word in nopunc.split() if  word.lower() not in stopwords.words('english')]#去除掉常用关键字
    return nopunc
#存储模型训练过程
pline=Pipeline([
    ('bow',TfidfVectorizer(analyzer=text_process)),#利用TF-IDF对训练数据文本中的单词进行计算得出权重关系
    ('classifier',MultinomialNB())])#申明朴素贝叶斯分类器

# print(type(message['message'].head().apply(text_process)))

pline.fit(msg_train, label_train)#训练
predict=pline.predict(msg_test)#预测
print(predict)
print('/n')
print(classification_report(predict,label_test))#评分

# bow_trans=TfidfVectorizer(analyzer=text_process).fit(message['message'])
# messages_tfidf = bow_trans.transform(message['message'])
# print(type(messages_tfidf))
# print(messages_tfidf)
# spam_detect_model = MultinomialNB().fit(messages_tfidf, message['label'])